import requests
from bs4 import BeautifulSoup
import time
import re

base_url = 'http://m.27baola.com'

headers = {'Accept': 'text/html, application/xhtml+xml, image/jxr, */*',
           'Accept - Encoding': 'gzip, deflate',
           'Accept-Language': 'zh-Hans-CN, zh-Hans; q=0.5',
           'Connection': 'Keep-Alive',
           'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063'}
imgurl_list = []
count = 0

for i in range(1, 57):
    url = 'http://m.27baola.com/gif/list_'+str(i)+'.html'

    try:
        r = requests.get(url)
        content = r.content
        soup = BeautifulSoup(content, "lxml")


        pa = soup.find('ul', attrs={'class': 'pic pic1'})
        imgset_list = pa.find_all('a')
        time.sleep(0.1)

        for imgset in imgset_list:
            imgset_url = base_url+imgset['href']

            try:
                rr = requests.get(imgset_url)
                ccontent = rr.content
                ssoup = BeautifulSoup(ccontent, "lxml")


                ndiv = ssoup.find('div', attrs={'class': 'showpage'})
                pagenum = int(re.sub("\D", "", ndiv.find_all('a')[1].text))

                div = ssoup.find('div', attrs={'class': 'imgString'})
                imgs = div.find_all('img')
                for img in imgs:
                    imgurl_list.append(img['src'])
                    count = count+1
                time.sleep(0.1)

                for pn in range(2, pagenum+1):
                    img_detail_url = imgset_url[:-5]+'_'+str(pn)+'.html'

                    try:
                        rrr = requests.get(img_detail_url)
                        cccontent = rrr.content
                        sssoup = BeautifulSoup(cccontent, "lxml")
                

                        ddiv = sssoup.find('div', attrs={'class': 'imgString'})
                        imgs = ddiv.find_all('img')
                        for img in imgs:
                            imgurl_list.append(img['src'])
                            count = count+1
                        print(str(count), end='\r')
                        time.sleep(0.1)
                    except Exception as e:
                        print(e)
                    finally:
                        pass
            except Exception as e:
                print(e)
            finally:
                pass
    except Exception as e:
        print(e)
    finally:
        pass

print('strat write to disk...')
with open('27baolagifurl.txt', mode='a+') as file:
    for imgurl in imgurl_list:
        file.write(imgurl+'\n')
